# -*- coding: utf-8 -*-
import scrapy
from Tencent.items import TencentItem


class TencentSpider(scrapy.Spider):
    name = 'tencent'

    # urls里可以同时放入多个网页
    # def start_requests(self):
    #     urls = [
    #         'http://www.gz.gov.cn/gzgov/snzc/common_list.shtml'
    #     ]
    #     for url in urls:
    #         yield scrapy.Request(url=url, callback=self.parse)

    baseURL = 'http://www.gz.gov.cn/gzgov/snzc/common_list_'
    offset = 1
    end = '.shtml'
    start_urls = ['http://www.gz.gov.cn/gzgov/snzc/common_list.shtml']

    def parse(self, response):
        node_list = response.xpath('//ul[@class="news_list"]/li')

        for node in node_list:
            item = TencentItem()

            title = node.xpath('./a/text()').extract()
            time = node.xpath('./span/text()').extract()
            link = node.xpath('./a/@href').extract()
            item['title'] = title[0]
            item['time'] = time[0]
            item['link'] = link[0]
            yield item

        if self.offset < 16:
            self.offset += 1
            url = self.baseURL + str(self.offset) + self.end
            yield scrapy.Request(url, callback=self.parse)
